# -*- coding: utf-8 -*-
"""
Created on Thu Oct 10 13:59:03 2013

@author: jkwong
"""

#PBAR_PrcoessStandardWidthImages.py

import PBAR_Zspec, PBAR_Cargo
reload(PBAR_Zspec)
import numpy as np
import os
import matplotlib.pyplot as plt
#from scipy import interpolate
from matplotlib import cm
from scipy import ndimage
import copy
import glob
#import json
import pickle


# Set useful plot variables
plotColors = ['r', 'b', 'g', 'm', 'c', 'y', 'k'] * 10
lineStyles = ['-', '-.', ':', '_', '|'] *  10
markerTypes = ['.', 'o', 'v', '^', '<', '>', '1', '2', '3', '4', 's', 'p', '*', 'h', 'H', '+', 'x', 'D', 'd']
markerTypes = markerTypes * 2
colorList = np.array(['r', 'b', 'g', 'm', 'c', 'y'])

# list of zpsec detector names - Z1 to Z136
ZspecNameList = np.array(['Z%d'%i for i in np.arange(1, 137)])
# list of bad detector numbers starting at 1
badZspecList = np.array([1,2,3,4,5,6,7,8,20,26,31,33,34,38,39,40,44,53,56,62,68,76,80,125,126,127,128,129,130,131,132,133,134,135,136])
# list of good detector numbers starting at 1
goodZspecList = np.array([i for i in np.arange(1, 137) if (i not in badZspecList)])
# list of bad detector names
badZspecNameList = ZspecNameList[(badZspecList-1)]
# list of good detector names
goodZspecNameList = np.array(['Z%d'%i for i in np.arange(1, 137) if (i not in badZspecList)])

badZspecIndices = badZspecList - 1
goodZspecIndices = goodZspecList - 1

badZspecMask = np.zeros(len(ZspecNameList))
badZspecMask[badZspecIndices] = True
badZspecMask = badZspecMask.astype(bool)

goodZspecMask = np.zeros(len(ZspecNameList))
goodZspecMask[goodZspecIndices] = True
goodZspecMask = goodZspecMask.astype(bool)

dataDir = r'E:\PBAR\data3b\BasicScansStandardWidth'
plotDir = r'E:\PBAR\data3b\BasicScansPlots'

# define parameters for standard width images
numberTimeSlices = 2000
preCargoTimeSlices = 70

acqTime = 1/60.
figureSize = (16, 10)

params = {'legend.fontsize': 8}
plt.rcParams.update(params)

#  wBest = np.array([-2.2806503 ,  2.26895887,  6.7898522 ])  # old 10/14/2013
        
wBest = np.array([-2.07779891,  1.88095226,  4.65429156])
pfitmean = np.array([ -3.45931698e-06,   1.80445624e-03,  -3.46939502e-01,  9.03548991e-01])
pfitsigma = np.array([ -2.90915823e-07,   2.71344856e-04,  -6.81517865e-02, 6.54989380e+00])

#################################
## DEFINE FILE NAMES

cargoConfigBaseDirList = glob.glob(os.path.join(dataDir, '*'))

#cargoConfigBaseDirList = cargoConfigBaseDirList[42:]
#cargoConfigBaseDirList =cargoConfigBaseDirList[-1:]

cargoConfigBaseDirList = cargoConfigBaseDirList[31:32]
#cargoConfigBaseDirList = cargoConfigBaseDirList[34:43]



for (dirIndex, cargoConfigBaseDir) in enumerate(cargoConfigBaseDirList):
    subDirList = glob.glob(os.path.join(cargoConfigBaseDir, '*'))
    for (subDirIndex, dataPath) in enumerate(subDirList):
        print(dataPath)

        a, b = os.path.split(dataPath)
        a, c = os.path.split(a)

        plotSaveDir = os.path.join(plotDir, c, b)
        
        # make paths if not present
        if not os.path.exists(os.path.join(plotDir, c)):
            os.mkdir(os.path.join(plotDir, c))

        if not os.path.exists(plotSaveDir):
            os.mkdir(plotSaveDir)
        
        # find the zspec scan number
        temp = glob.glob(os.path.join(dataPath, '*.npy'))
        # if file doesn't exist skip this set completely
        if temp == []:
            print('No npy file found.  Skipping.')
            continue
        
        for t in temp:
            (a, b) = os.path.split(t)
            if len(b) < 22:  # we need to do this now to avoid the cargoimage.npy
                a, filename = os.path.split(temp[0])
                zspecScanNumber = filename[0:4]
                filenameZspec = filename
                fullFilenameZspec = os.path.join(dataPath, filename)
                break
        
        # Get the cargo image name
        temp = glob.glob(os.path.join(dataPath, '*.cargoimage.npy'))
        # if file doesn't exist skip this set completely
        if temp == []:
            print('No cargoimage file found.  Skipping.')
            continue
        a, filename = os.path.split(temp[0])
        cargoScanNumber = filename[0:16]
        filenameCargo = filename    
        fullFilenameCargo = os.path.join(dataPath, filename)
        
        fullFilenameMarker = os.path.join(dataPath, filename.replace('cargoimage.npy', 'cargomarker'))
        acquisitionTime = acqTime
        
        ######################
        ## READ IN DATA
        
        # Read in marker files
        markerStandardWidth = PBAR_Cargo.ReadCargoMarker(fullFilenameMarker)
        
        # Read Zspec
        datStandardWidth = np.load(fullFilenameZspec)
        
        # Read Cargo
        datCargoStandardWidth= np.load(fullFilenameCargo)
        
        #############################
        ##  CALCULATE FEATURES
        
        discrim = []
        temp = {}
        energy = np.array([7.50E+00,1.05E+01,1.35E+01,1.65E+01,1.95E+01,2.25E+01,2.55E+01, \
            2.85E+01,3.15E+01,3.45E+01,3.75E+01,4.05E+01,4.35E+01,4.65E+01,4.95E+01,\
            5.25E+01,5.55E+01,5.85E+01,6.15E+01,6.45E+01,6.75E+01,7.05E+01,7.35E+01, \
            7.65E+01,7.95E+01,8.25E+01,8.55E+01,8.85E+01,9.15E+01,9.45E+01,9.75E+01,1.01E+02])
        
        discrim = PBAR_Zspec.CalculateFeaturesZspecBasic(datStandardWidth, energy)   

        # Calculate the dist, linear combination
        # make features array (num features X time slices X number of detectors)
        featuresMatrix = np.zeros((3, discrim['binMean'].shape[0], discrim['binMean'].shape[1]))
        featuresMatrix[0,:,:] = discrim['binMean']
        featuresMatrix[1,:,:] = discrim['binSTD']
        featuresMatrix[2,:,:] = discrim['multibin_20_ratio']
        
        discrim['dist0']  = featuresMatrix[0,:,:] *wBest[0] + featuresMatrix[1,:,:] *wBest[1] + featuresMatrix[2,:,:] *wBest[2]
#            # clean up bad values
#            cutt = np.isnan(dist0)
#            dist0[cutt] = 0.0
            
        # Cut in radiography image of standard size
#        cargoCountRange = np.array([0.01e8, 0.4e8]) # only consider parts of the zspec image that have rad below this value
        cargoCountRange = np.array([0, 0.4e8]) # only consider parts of the zspec image that have rad below this value
     
        cargoZspecStandardWidthMask = ( datCargoStandardWidth >= cargoCountRange[0] ) & \
            ( datCargoStandardWidth <= cargoCountRange[1] )
        
        # Clean up radiograph mask
        temp = cargoZspecStandardWidthMask
        # do some stuff
        temp = ndimage.binary_opening(temp)
        cargoZspecStandardWidthCleanedMask = temp

        # output a npy file for each distriminant
        for (keyIndex, k) in enumerate(discrim.keys()):
            f = fullFilenameZspec.replace('.npy', '_%s.npy' %k)
            np.save(f, discrim[k])
            print('Wrote: %s' % f)

        markerValues = []
        for i, mark in enumerate(markerStandardWidth):
            markTemp = copy.deepcopy(mark)
            
            # CENTER POINT
            temp = {}
            x_center = mark['x']
            y_center = mark['y']
            
            temp['x'] = x_center
            temp['y'] = y_center
            temp['xmean'] = x_center
            temp['ymean'] = y_center
            
            # cycle through all features in the discrim dictionary
            discrimTemp = {}            
            for (keyIndex, key) in enumerate(discrim.keys()):
                discrimTemp[key] = discrim[key][x_center, y_center]
            discrimTemp['cargoCount'] = datCargoStandardWidth[x_center, y_center]
            temp['discrim'] = discrimTemp
            discrimTemp['cargoCount'] = datCargoStandardWidth[x_center, y_center]           
            
            markTemp['center'] = temp
            
            
            #  CENTER BLOCK
            temp = {}
            x_center = mark['x']
            y_center = mark['y']
            temp['x'] = np.arange(-2, 3) + x_center
            temp['y'] = np.arange(-1, 2) + y_center
            temp['xmean'] = x_center
            temp['ymean'] = y_center
            
            # make the bounds of the box
            x_range = np.array((temp['x'][0], temp['x'][-1])) # left < right
            y_range = np.array((temp['y'][0], temp['y'][-1])) # top  < bottom
            
            xarray = np.arange(discrim['dist0'].shape[0])
            yarray = np.arange(discrim['dist0'].shape[1])
            
            xcut = (xarray >= x_range[0]) & (xarray <= x_range[1])
            ycut = (yarray >= y_range[0]) & (yarray <= y_range[1])
            
            discrimTemp = {}
            discrimMeanTemp = {}
            
            for (keyIndex, key) in enumerate(discrim.keys()):
                discrimTemp[key] = discrim[key][:,goodZspecMask & ycut][xcut,:].flatten()
                temp2 = discrim[key][:,goodZspecMask & ycut][xcut,:].flatten()
                cut = ~np.isnan(temp2) & ~np.isinf(temp2)
                discrimMeanTemp[key] = temp2[cut].mean()
            discrimTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut]
            discrimMeanTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut].flatten().mean()
            
            temp['discrim'] = discrimTemp
            temp['discrimMean'] = discrimMeanTemp
            markTemp['centerBlock'] = temp
            del temp2, temp, discrimTemp, discrimMeanTemp
            
            
            #  MARKER BOX
            temp = {}
            temp['x'] = np.arange(mark['rec_left'], mark['rec_right']+1)
            temp['y'] = np.arange(mark['rec_top'], mark['rec_bottom']+1)
            temp['xmean'] = temp['x'].mean()
            temp['ymean'] = temp['y'].mean()
            
            # make the bounds of the box
            x_range = np.array((temp['x'][0], temp['x'][-1])) # left < right
            y_range = np.array((temp['y'][0], temp['y'][-1])) # top  < bottom
            
            xarray = np.arange(discrim['dist0'].shape[0])
            yarray = np.arange(discrim['dist0'].shape[1])
            
            xcut = (xarray >= x_range[0]) & (xarray <= x_range[1])
            ycut = (yarray >= y_range[0]) & (yarray <= y_range[1])
            
            discrimTemp = {}
            discrimMeanTemp = {}
            
            for (keyIndex, key) in enumerate(discrim.keys()):
                discrimTemp[key] = discrim[key][:,goodZspecMask & ycut][xcut,:].flatten()
                temp2 = discrim[key][:,goodZspecMask & ycut][xcut,:].flatten()
                cut = ~np.isnan(temp2) & ~np.isinf(temp2)
                discrimMeanTemp[key] = temp2[cut].mean()
            discrimTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut]
            discrimMeanTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut].flatten().mean()
            
            temp['discrim'] = discrimTemp
            temp['discrimMean'] = discrimMeanTemp
            markTemp['block'] = temp
            del temp2, temp, discrimTemp, discrimMeanTemp
            
            
            #  MARKER BOX, only points in cargo mask
            temp = {}
            temp['x'] = np.arange(mark['rec_left'], mark['rec_right']+1)
            temp['y'] = np.arange(mark['rec_top'], mark['rec_bottom']+1)
            temp['xmean'] = temp['x'].mean()
            temp['ymean'] = temp['y'].mean()
            
            # make the bounds of the box
            x_range = np.array((temp['x'][0], temp['x'][-1])) # left < right
            y_range = np.array((temp['y'][0], temp['y'][-1])) # top  < bottom
            
            xarray = np.arange(discrim['dist0'].shape[0])
            yarray = np.arange(discrim['dist0'].shape[1])
            
            xcut = (xarray >= x_range[0]) & (xarray <= x_range[1])
            ycut = (yarray >= y_range[0]) & (yarray <= y_range[1])
            
            discrimTemp = {}
            discrimMeanTemp = {}
            
            maskCut = cargoZspecStandardWidthCleanedMask[:,goodZspecMask & ycut][xcut,:]    
            
            for (keyIndex, key) in enumerate(discrim.keys()):
                discrimTemp[key] = discrim[key][:,goodZspecMask & ycut][xcut,:][maskCut].flatten()
                temp2 = discrim[key][:,goodZspecMask & ycut][xcut,:][maskCut].flatten()
                cut = ~np.isnan(temp2) & ~np.isinf(temp2)
                discrimMeanTemp[key] = temp2[cut].mean()
            discrimTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut]
            discrimMeanTemp['cargoCount'] = datCargoStandardWidth[xcut, :][:, ycut].flatten().mean()
            
            temp['discrim'] = discrimTemp
            temp['discrimMean'] = discrimMeanTemp
            markTemp['blockMask'] = temp
            del temp2, temp, discrimTemp, discrimMeanTemp            
            
            
            markerValues.append(markTemp)
            
        # convert to json and dump
        fullFilenameMarkerValues = fullFilenameMarker.replace('cargomarker', 'cargomarkervalues')
        with open(fullFilenameMarkerValues, 'wb') as fid:
            pickle.dump(markerValues, fid)
            print('Write %s' %fullFilenameMarkerValues)
#        jsonText = json.dumps(markerValues)
#        with open(fullFilenameMarkerValues, 'wb') as fid:
#            fid.write(jsonText)
#            print('Write %s' %fullFilenameMarkerValues)
        
        ###################
        ## PLOTS 
        index = 0
        
        ## Standard Size Images ##
        
        ## Cargo standard size with markers
        # for zspec plot the total counts
        logIntensity = 1
        plotMarkers = True
        plt.figure(figsize = figureSize)
        plt.grid()
        
        if logIntensity:
            intensity = np.log(datCargoStandardWidth.T)
        else:
            intensity = datCargoStandardWidth.T
            
        plt.imshow(intensity, interpolation = 'nearest', aspect='auto', cmap = cm.Greys_r)
        plt.colorbar()
        plt.title(filenameCargo + ', Standard Size')
        plt.xlim((0, 2400))
        
        if plotMarkers:
            for i, mark in enumerate(markerStandardWidth):
                x = mark['rec_left'], mark['rec_right'], mark['rec_right'], mark['rec_left'], mark['rec_left']
                y = mark['rec_bottom'], mark['rec_bottom'], mark['rec_top'], mark['rec_top'], mark['rec_bottom']
                plt.plot(x, y, 'r')
                plt.plot(mark['x'], mark['y'], 'xr')
                if 'left' in mark:
                    plt.plot(mark['left']['x'], mark['left']['y'], 'gx')
                if 'right' in mark:
                    plt.plot(mark['right']['x'], mark['right']['y'], 'go')
                plt.text(mark['x'], mark['y'],  mark['target'], fontsize = 12, color = 'r')

        # Save image to file
        plotFullFilename = os.path.join(plotSaveDir, '%s_StandardSize.png' %filenameCargo)
        plt.savefig(plotFullFilename)        
        plt.close('all')
        

        # Zspec image, counts with markers        
        plt.figure(figsize = figureSize)
        plt.grid()
        featureName1 = 'count'
        t = plt.imshow(discrim[featureName1].T, interpolation = 'nearest', aspect='auto', cmap = cm.Greys_r)
        plt.colorbar()
        if plotMarkers:
            for i, mark in enumerate(markerStandardWidth):
                x = mark['rec_left'], mark['rec_right'], mark['rec_right'], mark['rec_left'], mark['rec_left']
                y = mark['rec_bottom'], mark['rec_bottom'], mark['rec_top'], mark['rec_top'], mark['rec_bottom']
                plt.plot(x, y, 'r')
                plt.plot(mark['x'], mark['y'], 'xr')
                if 'left' in mark:
                    plt.plot(mark['left']['x'], mark['left']['y'], 'gx')
                if 'right' in mark:
                    plt.plot(mark['right']['x'], mark['right']['y'], 'go')
                plt.text(mark['x'], mark['y'],  mark['target'], fontsize = 12, color = 'r')
        plt.title(filenameZspec + ', Standard Size')
        plt.xlim((0, 2400))
                
        # save image to file
        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_StandardSize.png' %(filenameZspec, featureName1))
        plt.savefig(plotFullFilename)
        plt.close('all')
                
        
        # Zspec dist image
        plt.figure(figsize = figureSize)
        plt.grid()
        featureName1 = 'dist0'
        t = plt.imshow(discrim[featureName1].T, interpolation = 'nearest', aspect='auto', cmap = cm.Greys_r)
        if plotMarkers:
            for i, mark in enumerate(markerStandardWidth):
                x = mark['rec_left'], mark['rec_right'], mark['rec_right'], mark['rec_left'], mark['rec_left']
                y = mark['rec_bottom'], mark['rec_bottom'], mark['rec_top'], mark['rec_top'], mark['rec_bottom']
                plt.plot(x, y, 'r')
                plt.plot(mark['x'], mark['y'], 'xr')
                if 'left' in mark:
                    plt.plot(mark['left']['x'], mark['left']['y'], 'gx')
                if 'right' in mark:
                    plt.plot(mark['right']['x'], mark['right']['y'], 'go')
                plt.text(mark['x'], mark['y'],  mark['target'], fontsize = 12, color = 'r')
        plt.colorbar()
        plt.title('%s, %s ' %(filenameZspec, featureName1))
        plt.xlim((0, 2200))
        
        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_StandardSize.png' %(filenameZspec, featureName1))
        plt.savefig(plotFullFilename)
        
        
        # Zspec dist image with mask
        plt.figure(figsize = figureSize)
        plt.grid()
        featureName1 = 'dist0'
        im = copy.copy(discrim[featureName1].T)
        minValue = im.min()
        im[~cargoZspecStandardWidthCleanedMask.T] = minValue
        t = plt.imshow(im, interpolation = 'nearest', aspect='auto', cmap = cm.Greys_r)
        if plotMarkers:
            for i, mark in enumerate(markerStandardWidth):
                x = mark['rec_left'], mark['rec_right'], mark['rec_right'], mark['rec_left'], mark['rec_left']
                y = mark['rec_bottom'], mark['rec_bottom'], mark['rec_top'], mark['rec_top'], mark['rec_bottom']
                plt.plot(x, y, 'r')
                plt.plot(mark['x'], mark['y'], 'xr')
                if 'left' in mark:
                    plt.plot(mark['left']['x'], mark['left']['y'], 'gx')
                if 'right' in mark:
                    plt.plot(mark['right']['x'], mark['right']['y'], 'go')
                plt.text(mark['x'], mark['y'],  mark['target'], fontsize = 12, color = 'r')
        
        plt.colorbar()
        plt.title('Linear Combination')
        plt.xlim((0, 2200))
        plt.show()

        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_Mask_StandardSize.png' %(filenameZspec, featureName1))
        plt.savefig(plotFullFilename)  
        plt.close('all')

        # Cargo mask
        plt.figure(figsize = figureSize)
        plt.grid()
        t = plt.imshow(cargoZspecStandardWidthCleanedMask.T, interpolation = 'nearest', aspect='auto', cmap = cm.Greys_r)
        if plotMarkers:
            for i, mark in enumerate(markerStandardWidth):
                x = mark['rec_left'], mark['rec_right'], mark['rec_right'], mark['rec_left'], mark['rec_left']
                y = mark['rec_bottom'], mark['rec_bottom'], mark['rec_top'], mark['rec_top'], mark['rec_bottom']
                plt.plot(x, y, 'r')
                plt.plot(mark['x'], mark['y'], 'xr')
                if 'left' in mark:
                    plt.plot(mark['left']['x'], mark['left']['y'], 'gx')
                if 'right' in mark:
                    plt.plot(mark['right']['x'], mark['right']['y'], 'go')
                plt.text(mark['x'], mark['y'],  mark['target'], fontsize = 12, color = 'r')
        
        plt.colorbar()
        plt.title('Cargo Mask: %d, %d' %(cargoCountRange[0], cargoCountRange[1]))
        plt.xlim((0, 2200))
        plt.show()

        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_CargoMask_StandardSize.png' %(filenameZspec, featureName1))
        plt.savefig(plotFullFilename)
        plt.close('all')



        ### Scatter Plots ###

        #  All data points and mask, and those in the boxes
        plt.figure(figsize = figureSize)
        plt.grid()
        index = 0
        feature1 = 'count'
        feature2 = 'dist0'
        # x, y for the whole image
        x1 = discrim[feature1][:,goodZspecIndices].flatten()
        y1 = discrim[feature2][:,goodZspecIndices].flatten()
        
        # reduce the mask to good detectors
        cut = cargoZspecStandardWidthCleanedMask[:, goodZspecIndices]
        # x values in mask, good detectors        
        x2 = discrim[feature1][:,goodZspecIndices][cut].flatten()
        # y values in mask, good detectors        
        y2 = discrim[feature2][:,goodZspecIndices][cut].flatten()
        
        plt.plot(x1, y1, '.k', alpha = 0.15, markersize = 10, label = 'All Points')
        plt.plot(x2, y2, '.b', alpha = 0.15, markersize = 10, label = 'In Mask')
        
        # plot the points in the markers
        for i, mark in enumerate(markerStandardWidth):
            x_range = np.array((mark['rec_left'], mark['rec_right'])) # left < right
            y_range = np.array((mark['rec_top'], mark['rec_bottom'])) # top  < bottom
            xarray = np.arange(discrim[feature1].shape[0])
            yarray = np.arange(discrim[feature1].shape[1])
            xcut = (xarray > x_range[0]) & (xarray < x_range[1])
            ycut = (yarray > y_range[0]) & (yarray < y_range[1])
            
            x = discrim[feature1][:,goodZspecMask & ycut][xcut,:].flatten()
            y = discrim[feature2][:,goodZspecMask & ycut][xcut,:].flatten()
            
        #            plt.plot(x, y, '.', markersize = 10, alpha  = 0.3, marker = markerTypes[i/6], color = colorList[i%6], label = mark['target'])
            # high density stuff
            try:
                if mark['target'][0] == 'S':
                    plt.plot(x, y, 'db', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'W':
                    plt.plot(x, y, 'vb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'D':
                    plt.plot(x, y, 'sb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'P':
                    plt.plot(x, y, '*b', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(x, y, 'or', markersize = 8, alpha  = 0.3, label = mark['target'])
                else:
                    plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])            
            except:
                plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            # high density stuff
        #    xx, yy = markerValues[i]['centerBlock']['discrimMean'][feature1], markerValues[i]['centerBlock']['discrimMean'][feature2]
            xx, yy = markerValues[i]['center']['discrim'][feature1], markerValues[i]['center']['discrim'][feature2]
        
            try:
                if markerValues[i]['target'][0] == 'S':
                    plt.plot(xx, yy, 'db', markersize = 18, alpha  = 1.0, label = mark['target'] +' Center')
                elif mark['target'][0] == 'W':
                    plt.plot(xx, yy, 'vb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'D':
                    plt.plot(xx, yy, 'sb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'P':
                    plt.plot(xx, yy, '*b', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(xx, yy, 'or', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                else:
                    plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
            except:
                plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
        
        plt.title(filenameZspec + ', Linear Combination Vs Counts')
        plt.legend()
        plt.xlabel(feature1)
        plt.ylabel(feature2)
        plt.xlim((0, 500))
        plt.ylim((-100, 100))
        plt.show()

        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_%s_AllPoints_Mask_Markers.png' %(filenameZspec, featureName1, feature2))
        plt.savefig(plotFullFilename)
        plt.close('all')

        #  Data points in mask only 
        removeOffset = 0
        
        plt.figure(figsize = figureSize)
        plt.grid()
        feature1 = 'count'
        feature2 = 'dist0'
        # reduce the mask to good detectors
        cut = cargoZspecStandardWidthCleanedMask[:, goodZspecIndices]
        # x values in mask, good detectors        
        x2 = discrim[feature1][:,goodZspecIndices][cut].flatten()
        # y values in mask, good detectors        
        y2 = discrim[feature2][:,goodZspecIndices][cut].flatten()
        if removeOffset:
            y2 = y2 - np.polyval(pfitmean, x2)
        plt.plot(x2, y2, '.k', alpha = 0.15, markersize = 10, label = 'In Mask')
        
        # plot the points in the markers
        for i, mark in enumerate(markerStandardWidth):
            x_range = np.array((mark['rec_left'], mark['rec_right'])) # left < right
            y_range = np.array((mark['rec_top'], mark['rec_bottom'])) # top  < bottom
            xarray = np.arange(discrim[feature1].shape[0])
            yarray = np.arange(discrim[feature1].shape[1])
            xcut = (xarray > x_range[0]) & (xarray < x_range[1])
            ycut = (yarray > y_range[0]) & (yarray < y_range[1])
            x = discrim[feature1][:,goodZspecMask & ycut][xcut,:].flatten()
            y = discrim[feature2][:,goodZspecMask & ycut][xcut,:].flatten()
        #            plt.plot(x, y, '.', markersize = 15, alpha  = 0.3, marker = markerTypes[i/6], color = colorList[i%6], label = mark['target'])
            if removeOffset:
                y = y - np.polyval(pfitmean, x)
                
            try:
                # high density stuff
                if mark['target'][0] == 'S':
                    plt.plot(x, y, 'db', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'W':
                    plt.plot(x, y, 'vb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'D':
                    plt.plot(x, y, 'sb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'P':
                    plt.plot(x, y, '*b', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(x, y, 'or', markersize = 8, alpha  = 0.3, label = mark['target'])
                else:
                    plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            except:
                plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            
            # high density stuff
            #    xx, yy = markerValues[i]['centerBlock']['discrimMean'][feature1], markerValues[i]['centerBlock']['discrimMean'][feature2]
            xx, yy = markerValues[i]['center']['discrim'][feature1], markerValues[i]['center']['discrim'][feature2]
            
            if removeOffset:
                yy = yy - np.polyval(pfitmean, xx)
            try:
                if markerValues[i]['target'][0] == 'S':
                    plt.plot(xx, yy, 'db', markersize = 18, alpha  = 1.0, label = mark['target'] +' Center')
                elif mark['target'][0] == 'W':
                    plt.plot(xx, yy, 'vb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'D':
                    plt.plot(xx, yy, 'sb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'P':
                    plt.plot(xx, yy, '*b', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(xx, yy, 'or', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                else:
                    plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
            except:
                plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                
        plt.title(filenameZspec + ', Linear Combination Vs Counts, Special Points Not Necessarily in Mask')
        plt.legend()
        plt.xlabel(feature1)
        plt.ylabel(feature2)
        plt.xlim((0, 500))
        plt.ylim((-100, 100))
        plt.show()
        
        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_%s_Mask_Markers.png' %(filenameZspec, featureName1, feature2))
        plt.savefig(plotFullFilename)  
        plt.close('all')
    
        
        #  Data points in mask, box points can contain points outside mask
        removeOffset = 1
        
        plt.figure(figsize = figureSize)
        plt.grid()
        feature1 = 'count'        
        feature2 = 'dist0'
        # reduce the mask to good detectors
        cut = cargoZspecStandardWidthCleanedMask[:, goodZspecIndices]
        # x values in mask, good detectors        
        x2 = discrim[feature1][:,goodZspecIndices][cut].flatten()
        # y values in mask, good detectors        
        y2 = discrim[feature2][:,goodZspecIndices][cut].flatten()
        if removeOffset:
            y2 = y2 - np.polyval(pfitmean, x2)
        plt.plot(x2, y2, '.k', alpha = 0.15, markersize = 10, label = 'In Mask')
        
        # plot the points in the markers
        for i, mark in enumerate(markerStandardWidth):
            x_range = np.array((mark['rec_left'], mark['rec_right'])) # left < right
            y_range = np.array((mark['rec_top'], mark['rec_bottom'])) # top  < bottom
            xarray = np.arange(discrim[feature1].shape[0])
            yarray = np.arange(discrim[feature1].shape[1])
            xcut = (xarray > x_range[0]) & (xarray < x_range[1])
            ycut = (yarray > y_range[0]) & (yarray < y_range[1])
            x = discrim[feature1][:,goodZspecMask & ycut][xcut,:].flatten()
            y = discrim[feature2][:,goodZspecMask & ycut][xcut,:].flatten()
        #            plt.plot(x, y, '.', markersize = 15, alpha  = 0.3, marker = markerTypes[i/6], color = colorList[i%6], label = mark['target'])
            if removeOffset:
                y = y - np.polyval(pfitmean, x)
            try:
                # high density stuff
                if mark['target'][0] == 'S':
                    plt.plot(x, y, 'db', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'W':
                    plt.plot(x, y, 'vb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'D':
                    plt.plot(x, y, 'sb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'P':
                    plt.plot(x, y, '*b', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(x, y, 'or', markersize = 8, alpha  = 0.3, label = mark['target'])
                else:
                    plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            except:
                plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            
            # high density stuff
            #    xx, yy = markerValues[i]['centerBlock']['discrimMean'][feature1], markerValues[i]['centerBlock']['discrimMean'][feature2]
            xx, yy = markerValues[i]['center']['discrim'][feature1], markerValues[i]['center']['discrim'][feature2]
            
            if removeOffset:
                yy = yy - np.polyval(pfitmean, xx)
            try:
                if markerValues[i]['target'][0] == 'S':
                    plt.plot(xx, yy, 'db', markersize = 18, alpha  = 1.0, label = mark['target'] +' Center')
                elif mark['target'][0] == 'W':
                    plt.plot(xx, yy, 'vb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'D':
                    plt.plot(xx, yy, 'sb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'P':
                    plt.plot(xx, yy, '*b', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(xx, yy, 'or', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                else:
                    plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
            except:
                plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
        plt.title(filenameZspec + ', Linear Combination Vs Counts, Special Points Not Necessarily in Mask')
        plt.legend()
        plt.xlabel(feature1)
        plt.ylabel(feature2)
        plt.xlim((0, 500))
        plt.ylim((-100, 100))
        plt.show()
        
        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_%s_Mask_Markers_OffsetRemoved.png' %(filenameZspec, featureName1, feature2))
        plt.savefig(plotFullFilename)  
        plt.close('all')
        
        
        #  Data points in mask, box points can contain points outside mask
        removeOffset = 1
        
        plt.figure(figsize = figureSize)
        plt.grid()
        feature1 = 'count'        
        feature2 = 'dist0'
        # reduce the mask to good detectors
        cut = cargoZspecStandardWidthCleanedMask[:, goodZspecIndices]
        # x values in mask, good detectors        
        x2 = discrim[feature1][:,goodZspecIndices][cut].flatten()
        # y values in mask, good detectors        
        y2 = discrim[feature2][:,goodZspecIndices][cut].flatten()
        if removeOffset:
            y2 = y2 - np.polyval(pfitmean, x2)
        plt.plot(x2, y2, '.k', alpha = 0.15, markersize = 10, label = 'In Mask')
        
        # plot the points in the markers
        for i, mark in enumerate(markerStandardWidth):
            x_range = np.array((mark['rec_left'], mark['rec_right'])) # left < right
            y_range = np.array((mark['rec_top'], mark['rec_bottom'])) # top  < bottom
            xarray = np.arange(discrim[feature1].shape[0])
            yarray = np.arange(discrim[feature1].shape[1])
            xcut = (xarray > x_range[0]) & (xarray < x_range[1])
            ycut = (yarray > y_range[0]) & (yarray < y_range[1])

            # only includ points in Box AND Mask
            cut = cargoZspecStandardWidthCleanedMask[:,goodZspecMask & ycut][xcut,:]

            x = discrim[feature1][:,goodZspecMask & ycut][xcut,:][cut].flatten()
            y = discrim[feature2][:,goodZspecMask & ycut][xcut,:][cut].flatten()
            
        #            plt.plot(x, y, '.', markersize = 15, alpha  = 0.3, marker = markerTypes[i/6], color = colorList[i%6], label = mark['target'])
            if removeOffset:
                y = y - np.polyval(pfitmean, x)
            try:
                # high density stuff
                if mark['target'][0] == 'S':
                    plt.plot(x, y, 'db', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'W':
                    plt.plot(x, y, 'vb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'D':
                    plt.plot(x, y, 'sb', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'P':
                    plt.plot(x, y, '*b', markersize = 8, alpha  = 0.3, label = mark['target'])
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(x, y, 'or', markersize = 8, alpha  = 0.3, label = mark['target'])
                else:
                    plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            except:
                plt.plot(x, y, 'om', markersize = 8, alpha  = 0.3, label = mark['target'])
            
            # high density stuff
            #    xx, yy = markerValues[i]['centerBlock']['discrimMean'][feature1], markerValues[i]['centerBlock']['discrimMean'][feature2]
            xx, yy = markerValues[i]['center']['discrim'][feature1], markerValues[i]['center']['discrim'][feature2]
            
            if removeOffset:
                yy = yy - np.polyval(pfitmean, xx)
            try:
                if markerValues[i]['target'][0] == 'S':
                    plt.plot(xx, yy, 'db', markersize = 18, alpha  = 1.0, label = mark['target'] +' Center')
                elif mark['target'][0] == 'W':
                    plt.plot(xx, yy, 'vb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'D':
                    plt.plot(xx, yy, 'sb', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'P':
                    plt.plot(xx, yy, '*b', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                elif mark['target'][0] == 'F':# low density stuff
                    plt.plot(xx, yy, 'or', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
                else:
                    plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
            except:
                plt.plot(xx, yy, 'om', markersize = 18, alpha  = 1.0, label = mark['target']+' Center')
        plt.title(filenameZspec + ', Linear Combination Vs Counts, Special Points Not Necessarily in Mask')
        plt.legend()
        plt.xlabel(feature1)
        plt.ylabel(feature2)
        plt.xlim((0, 500))
        plt.ylim((-100, 100))
        plt.show()
        
        plotFullFilename = os.path.join(plotSaveDir, '%s_%s_%s_MaskOnly_Markers_OffsetRemoved.png' %(filenameZspec, featureName1, feature2))
        plt.savefig(plotFullFilename)  
        plt.close('all')
        